{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多种格式数据加载、处理与存储"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实际的场景中，我们会在不同的地方遇到各种不同的数据格式（比如大家熟悉的csv与txt，比如网页HTML格式，比如XML格式），我们来一起看看python如何和这些格式的数据打交道"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "from numpy.random import randn\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(12345)\n",
    "plt.rc('figure', figsize = (10, 6))\n",
    "from pandas import Series,DataFrame\n",
    "import pandas as pd\n",
    "np.set_printoptions(precision=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'C:\\\\Users\\\\jiangcheng\\\\Documents\\\\Python Scripts\\\\python_learn\\\\module0319'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.各式各样的文本数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b   c   d message\n",
       "0  1   2   3   4   hello\n",
       "1  5   6   7   8   world\n",
       "2  9  10  11  12     foo"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'a': [1, 5, 9],'b': [2, 6, 10],'c' : [3, 7, 11],'d': [4, 8, 12],'message': ['hello', 'world', 'foo']})\n",
    "df\n",
    "#df.to_csv('./4/data/data1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b   c   d message\n",
       "0  1   2   3   4   hello\n",
       "1  5   6   7   8   world\n",
       "2  9  10  11  12     foo"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# !cat ./4/data/data1.csv\n",
    "df = pd.read_csv('./4/data/data1.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b   c   d message\n",
       "0  1   2   3   4   hello\n",
       "1  5   6   7   8   world\n",
       "2  9  10  11  12     foo"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_table('./4/data/data1.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0   1   2   3      4\n",
       "0  1   2   3   4  hello\n",
       "1  5   6   7   8  world\n",
       "2  9  10  11  12    foo"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# !cat ./4/data/data2.csv\n",
    "pd.read_csv('./4/data/data2.csv', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b   c   d message\n",
       "0  1   2   3   4   hello\n",
       "1  5   6   7   8   world\n",
       "2  9  10  11  12     foo"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('./4/data/data2.csv',names = ['a', 'b', 'c', 'd', 'message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>message</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>hello</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>world</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>foo</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         a   b   c   d\n",
       "message               \n",
       "hello    1   2   3   4\n",
       "world    5   6   7   8\n",
       "foo      9  10  11  12"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = ['a', 'b', 'c', 'd', 'message']\n",
    "pd.read_csv('./4/data/data2.csv', names = names, index_col = 'message')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>value1</th>\n",
       "      <th>value2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key1</th>\n",
       "      <th>key2</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">one</th>\n",
       "      <th>a</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">two</th>\n",
       "      <th>a</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b</th>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c</th>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           value1  value2\n",
       "key1 key2                \n",
       "one  a          1       2\n",
       "     b          3       4\n",
       "     c          5       6\n",
       "     d          7       8\n",
       "two  a          9      10\n",
       "     b         11      12\n",
       "     c         13      14\n",
       "     d         15      16"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#!cat ./4/data/csv_mindex.csv\n",
    "parsed = pd.read_csv('./4/data/csv_mindex.csv', index_col = ['key1', 'key2'])\n",
    "parsed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[',A,B,C\\n',\n",
       " 'aaa,-0.264438,-1.026059,-0.619500\\n',\n",
       " 'bbb,0.927272,0.302904,-0.032399\\n',\n",
       " 'ccc,-0.264273,-0.386314,-0.217601\\n',\n",
       " 'ddd,-0.871858,-0.348382,1.100491']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(open('./4/data/data3.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aaa</td>\n",
       "      <td>-0.264438</td>\n",
       "      <td>-1.026059</td>\n",
       "      <td>-0.619500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bbb</td>\n",
       "      <td>0.927272</td>\n",
       "      <td>0.302904</td>\n",
       "      <td>-0.032399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ccc</td>\n",
       "      <td>-0.264273</td>\n",
       "      <td>-0.386314</td>\n",
       "      <td>-0.217601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ddd</td>\n",
       "      <td>-0.871858</td>\n",
       "      <td>-0.348382</td>\n",
       "      <td>1.100491</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0          1          2          3\n",
       "0  NaN          A          B          C\n",
       "1  aaa  -0.264438  -1.026059  -0.619500\n",
       "2  bbb   0.927272   0.302904  -0.032399\n",
       "3  ccc  -0.264273  -0.386314  -0.217601\n",
       "4  ddd  -0.871858  -0.348382   1.100491"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#result = pd.read_table('./4/data/data3.txt', sep='\\s+')#\\s+ 表示为若干个空格、tab等\n",
    "result = pd.read_table('./4/data/data3.txt', sep=',', header = None)#\\s+ 表示为若干个空格、tab等\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b   c   d message\n",
       "0  1   2   3   4   hello\n",
       "1  5   6   7   8   world\n",
       "2  9  10  11  12     foo"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# !cat ./4/data/data4.csv\n",
    "pd.read_csv('./4/data/data4.csv',skiprows=[0, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>something</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   something      a      b      c      d  message\n",
       "0      False  False  False  False  False     True\n",
       "1      False  False  False   True  False    False\n",
       "2      False  False  False  False  False    False"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# !cat ./4/data/data5.csv\n",
    "result = pd.read_csv('./4/data/data5.csv')\n",
    "result\n",
    "pd.isnull(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>something</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>three</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  something  a   b     c   d message\n",
       "0       one  1   2   3.0   4     NaN\n",
       "1       two  5   6   NaN   8   world\n",
       "2     three  9  10  11.0  12     foo"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.read_csv('./4/data/data5.csv', na_values = ['NULL'])\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>something</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>three</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  something  a   b     c   d message\n",
       "0       one  1   2   3.0   4     NaN\n",
       "1       two  5   6   NaN   8   world\n",
       "2     three  9  10  11.0  12     NaN"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentinels = {'message':['foo', 'NA'], 'somthing': ['two']}\n",
    "pd.read_csv('./4/data/data5.csv',na_values = sentinels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 分片/块读取文本数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.204708</td>\n",
       "      <td>0.478943</td>\n",
       "      <td>-0.519439</td>\n",
       "      <td>-0.555730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.965781</td>\n",
       "      <td>1.393406</td>\n",
       "      <td>0.092908</td>\n",
       "      <td>0.281746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.769023</td>\n",
       "      <td>1.246435</td>\n",
       "      <td>1.007189</td>\n",
       "      <td>-1.296221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.274992</td>\n",
       "      <td>0.228913</td>\n",
       "      <td>1.352917</td>\n",
       "      <td>0.886429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-2.001637</td>\n",
       "      <td>-0.371843</td>\n",
       "      <td>1.669025</td>\n",
       "      <td>-0.438570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.539741</td>\n",
       "      <td>0.476985</td>\n",
       "      <td>3.248944</td>\n",
       "      <td>-1.021228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.577087</td>\n",
       "      <td>0.124121</td>\n",
       "      <td>0.302614</td>\n",
       "      <td>0.523772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000940</td>\n",
       "      <td>1.343810</td>\n",
       "      <td>-0.713544</td>\n",
       "      <td>-0.831154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-2.370232</td>\n",
       "      <td>-1.860761</td>\n",
       "      <td>-0.860757</td>\n",
       "      <td>0.560145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-1.265934</td>\n",
       "      <td>0.119827</td>\n",
       "      <td>-1.063512</td>\n",
       "      <td>0.332883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-2.359419</td>\n",
       "      <td>-0.199543</td>\n",
       "      <td>-1.541996</td>\n",
       "      <td>-0.970736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-1.307030</td>\n",
       "      <td>0.286350</td>\n",
       "      <td>0.377984</td>\n",
       "      <td>-0.753887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.331286</td>\n",
       "      <td>1.349742</td>\n",
       "      <td>0.069877</td>\n",
       "      <td>0.246674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.011862</td>\n",
       "      <td>1.004812</td>\n",
       "      <td>1.327195</td>\n",
       "      <td>-0.919262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-1.549106</td>\n",
       "      <td>0.022185</td>\n",
       "      <td>0.758363</td>\n",
       "      <td>-0.660524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.862580</td>\n",
       "      <td>-0.010032</td>\n",
       "      <td>0.050009</td>\n",
       "      <td>0.670216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.852965</td>\n",
       "      <td>-0.955869</td>\n",
       "      <td>-0.023493</td>\n",
       "      <td>-2.304234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.652469</td>\n",
       "      <td>-1.218302</td>\n",
       "      <td>-1.332610</td>\n",
       "      <td>1.074623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.723642</td>\n",
       "      <td>0.690002</td>\n",
       "      <td>1.001543</td>\n",
       "      <td>-0.503087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-0.622274</td>\n",
       "      <td>-0.921169</td>\n",
       "      <td>-0.726213</td>\n",
       "      <td>0.222896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.051316</td>\n",
       "      <td>-1.157719</td>\n",
       "      <td>0.816707</td>\n",
       "      <td>0.433610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.010737</td>\n",
       "      <td>1.824875</td>\n",
       "      <td>-0.997518</td>\n",
       "      <td>0.850591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-0.131578</td>\n",
       "      <td>0.912414</td>\n",
       "      <td>0.188211</td>\n",
       "      <td>2.169461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-0.114928</td>\n",
       "      <td>2.003697</td>\n",
       "      <td>0.029610</td>\n",
       "      <td>0.795253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.118110</td>\n",
       "      <td>-0.748532</td>\n",
       "      <td>0.584970</td>\n",
       "      <td>0.152677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-1.565657</td>\n",
       "      <td>-0.562540</td>\n",
       "      <td>-0.032664</td>\n",
       "      <td>-0.929006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-0.482573</td>\n",
       "      <td>-0.036264</td>\n",
       "      <td>1.095390</td>\n",
       "      <td>0.980928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.589488</td>\n",
       "      <td>1.581700</td>\n",
       "      <td>-0.528735</td>\n",
       "      <td>0.457002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.929969</td>\n",
       "      <td>-1.569271</td>\n",
       "      <td>-1.022487</td>\n",
       "      <td>-0.402827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.220487</td>\n",
       "      <td>-0.193401</td>\n",
       "      <td>0.669158</td>\n",
       "      <td>-1.648985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9970</th>\n",
       "      <td>-0.515691</td>\n",
       "      <td>1.244901</td>\n",
       "      <td>1.302043</td>\n",
       "      <td>-1.034539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9971</th>\n",
       "      <td>-0.597495</td>\n",
       "      <td>1.385267</td>\n",
       "      <td>-0.763196</td>\n",
       "      <td>1.990051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9972</th>\n",
       "      <td>1.914450</td>\n",
       "      <td>0.382984</td>\n",
       "      <td>0.809112</td>\n",
       "      <td>-1.745766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9973</th>\n",
       "      <td>-1.281518</td>\n",
       "      <td>0.008116</td>\n",
       "      <td>-1.212094</td>\n",
       "      <td>-0.635271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9974</th>\n",
       "      <td>-1.141587</td>\n",
       "      <td>2.319184</td>\n",
       "      <td>-1.294746</td>\n",
       "      <td>-0.252059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9975</th>\n",
       "      <td>0.739082</td>\n",
       "      <td>0.966051</td>\n",
       "      <td>-1.178064</td>\n",
       "      <td>0.535442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9976</th>\n",
       "      <td>-0.051060</td>\n",
       "      <td>-0.312800</td>\n",
       "      <td>1.618086</td>\n",
       "      <td>2.258698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9977</th>\n",
       "      <td>-1.452134</td>\n",
       "      <td>1.808934</td>\n",
       "      <td>2.480793</td>\n",
       "      <td>0.926589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9978</th>\n",
       "      <td>-1.453201</td>\n",
       "      <td>0.369264</td>\n",
       "      <td>1.018868</td>\n",
       "      <td>-1.408517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9979</th>\n",
       "      <td>0.060708</td>\n",
       "      <td>-1.048864</td>\n",
       "      <td>-0.611300</td>\n",
       "      <td>0.372017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9980</th>\n",
       "      <td>1.301401</td>\n",
       "      <td>-1.171101</td>\n",
       "      <td>-1.461861</td>\n",
       "      <td>-0.269016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9981</th>\n",
       "      <td>1.124364</td>\n",
       "      <td>-0.900691</td>\n",
       "      <td>-0.949605</td>\n",
       "      <td>-0.845981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9982</th>\n",
       "      <td>-2.051208</td>\n",
       "      <td>0.775944</td>\n",
       "      <td>0.985752</td>\n",
       "      <td>-1.302367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9983</th>\n",
       "      <td>-0.739562</td>\n",
       "      <td>0.875037</td>\n",
       "      <td>-0.521750</td>\n",
       "      <td>0.275383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9984</th>\n",
       "      <td>-0.323421</td>\n",
       "      <td>-0.832905</td>\n",
       "      <td>-1.423236</td>\n",
       "      <td>0.957248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9985</th>\n",
       "      <td>1.586598</td>\n",
       "      <td>-1.465278</td>\n",
       "      <td>-1.184578</td>\n",
       "      <td>0.515831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9986</th>\n",
       "      <td>-0.110349</td>\n",
       "      <td>0.532887</td>\n",
       "      <td>0.332079</td>\n",
       "      <td>0.349169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9987</th>\n",
       "      <td>0.278290</td>\n",
       "      <td>-0.860830</td>\n",
       "      <td>0.505153</td>\n",
       "      <td>0.817113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9988</th>\n",
       "      <td>-0.434324</td>\n",
       "      <td>-1.897628</td>\n",
       "      <td>0.565004</td>\n",
       "      <td>1.508078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9989</th>\n",
       "      <td>-0.370955</td>\n",
       "      <td>-0.777062</td>\n",
       "      <td>-1.072253</td>\n",
       "      <td>-0.007181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9990</th>\n",
       "      <td>1.000089</td>\n",
       "      <td>1.167074</td>\n",
       "      <td>-0.188872</td>\n",
       "      <td>1.099893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9991</th>\n",
       "      <td>0.202083</td>\n",
       "      <td>0.626895</td>\n",
       "      <td>-0.562755</td>\n",
       "      <td>0.851409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9992</th>\n",
       "      <td>0.947399</td>\n",
       "      <td>0.137731</td>\n",
       "      <td>-1.341261</td>\n",
       "      <td>1.594911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9993</th>\n",
       "      <td>0.241749</td>\n",
       "      <td>0.613748</td>\n",
       "      <td>0.270834</td>\n",
       "      <td>0.915012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9994</th>\n",
       "      <td>2.825056</td>\n",
       "      <td>1.785693</td>\n",
       "      <td>-0.068129</td>\n",
       "      <td>-1.662357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0.481378</td>\n",
       "      <td>-0.735848</td>\n",
       "      <td>0.426780</td>\n",
       "      <td>-0.616316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>0.400465</td>\n",
       "      <td>-0.173379</td>\n",
       "      <td>-0.784637</td>\n",
       "      <td>0.326710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>-0.847900</td>\n",
       "      <td>-0.462586</td>\n",
       "      <td>-0.574423</td>\n",
       "      <td>0.632782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>1.299831</td>\n",
       "      <td>1.760948</td>\n",
       "      <td>2.089599</td>\n",
       "      <td>0.091444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>0.061865</td>\n",
       "      <td>0.781684</td>\n",
       "      <td>-0.045915</td>\n",
       "      <td>-0.773583</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           one       two     three      four\n",
       "0    -0.204708  0.478943 -0.519439 -0.555730\n",
       "1     1.965781  1.393406  0.092908  0.281746\n",
       "2     0.769023  1.246435  1.007189 -1.296221\n",
       "3     0.274992  0.228913  1.352917  0.886429\n",
       "4    -2.001637 -0.371843  1.669025 -0.438570\n",
       "5    -0.539741  0.476985  3.248944 -1.021228\n",
       "6    -0.577087  0.124121  0.302614  0.523772\n",
       "7     0.000940  1.343810 -0.713544 -0.831154\n",
       "8    -2.370232 -1.860761 -0.860757  0.560145\n",
       "9    -1.265934  0.119827 -1.063512  0.332883\n",
       "10   -2.359419 -0.199543 -1.541996 -0.970736\n",
       "11   -1.307030  0.286350  0.377984 -0.753887\n",
       "12    0.331286  1.349742  0.069877  0.246674\n",
       "13   -0.011862  1.004812  1.327195 -0.919262\n",
       "14   -1.549106  0.022185  0.758363 -0.660524\n",
       "15    0.862580 -0.010032  0.050009  0.670216\n",
       "16    0.852965 -0.955869 -0.023493 -2.304234\n",
       "17   -0.652469 -1.218302 -1.332610  1.074623\n",
       "18    0.723642  0.690002  1.001543 -0.503087\n",
       "19   -0.622274 -0.921169 -0.726213  0.222896\n",
       "20    0.051316 -1.157719  0.816707  0.433610\n",
       "21    1.010737  1.824875 -0.997518  0.850591\n",
       "22   -0.131578  0.912414  0.188211  2.169461\n",
       "23   -0.114928  2.003697  0.029610  0.795253\n",
       "24    0.118110 -0.748532  0.584970  0.152677\n",
       "25   -1.565657 -0.562540 -0.032664 -0.929006\n",
       "26   -0.482573 -0.036264  1.095390  0.980928\n",
       "27   -0.589488  1.581700 -0.528735  0.457002\n",
       "28    0.929969 -1.569271 -1.022487 -0.402827\n",
       "29    0.220487 -0.193401  0.669158 -1.648985\n",
       "...        ...       ...       ...       ...\n",
       "9970 -0.515691  1.244901  1.302043 -1.034539\n",
       "9971 -0.597495  1.385267 -0.763196  1.990051\n",
       "9972  1.914450  0.382984  0.809112 -1.745766\n",
       "9973 -1.281518  0.008116 -1.212094 -0.635271\n",
       "9974 -1.141587  2.319184 -1.294746 -0.252059\n",
       "9975  0.739082  0.966051 -1.178064  0.535442\n",
       "9976 -0.051060 -0.312800  1.618086  2.258698\n",
       "9977 -1.452134  1.808934  2.480793  0.926589\n",
       "9978 -1.453201  0.369264  1.018868 -1.408517\n",
       "9979  0.060708 -1.048864 -0.611300  0.372017\n",
       "9980  1.301401 -1.171101 -1.461861 -0.269016\n",
       "9981  1.124364 -0.900691 -0.949605 -0.845981\n",
       "9982 -2.051208  0.775944  0.985752 -1.302367\n",
       "9983 -0.739562  0.875037 -0.521750  0.275383\n",
       "9984 -0.323421 -0.832905 -1.423236  0.957248\n",
       "9985  1.586598 -1.465278 -1.184578  0.515831\n",
       "9986 -0.110349  0.532887  0.332079  0.349169\n",
       "9987  0.278290 -0.860830  0.505153  0.817113\n",
       "9988 -0.434324 -1.897628  0.565004  1.508078\n",
       "9989 -0.370955 -0.777062 -1.072253 -0.007181\n",
       "9990  1.000089  1.167074 -0.188872  1.099893\n",
       "9991  0.202083  0.626895 -0.562755  0.851409\n",
       "9992  0.947399  0.137731 -1.341261  1.594911\n",
       "9993  0.241749  0.613748  0.270834  0.915012\n",
       "9994  2.825056  1.785693 -0.068129 -1.662357\n",
       "9995  0.481378 -0.735848  0.426780 -0.616316\n",
       "9996  0.400465 -0.173379 -0.784637  0.326710\n",
       "9997 -0.847900 -0.462586 -0.574423  0.632782\n",
       "9998  1.299831  1.760948  2.089599  0.091444\n",
       "9999  0.061865  0.781684 -0.045915 -0.773583\n",
       "\n",
       "[10000 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = pd.DataFrame(np.random.randn(10000 * 4).reshape(10000,4),columns = ['one', 'two', 'three', 'four'])\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        0\n",
       "1        9\n",
       "2       16\n",
       "3       25\n",
       "4       25\n",
       "5       13\n",
       "6       13\n",
       "7        4\n",
       "8        1\n",
       "9        1\n",
       "10       4\n",
       "11       1\n",
       "12      16\n",
       "13       1\n",
       "14      13\n",
       "15      25\n",
       "16      25\n",
       "17       1\n",
       "18      13\n",
       "19       9\n",
       "20       1\n",
       "21      13\n",
       "22      13\n",
       "23       0\n",
       "24       1\n",
       "25      28\n",
       "26       4\n",
       "27       9\n",
       "28      25\n",
       "29      16\n",
       "        ..\n",
       "9970     1\n",
       "9971     9\n",
       "9972     9\n",
       "9973    16\n",
       "9974     0\n",
       "9975    28\n",
       "9976    25\n",
       "9977     0\n",
       "9978     9\n",
       "9979     9\n",
       "9980    25\n",
       "9981     0\n",
       "9982    13\n",
       "9983    13\n",
       "9984    28\n",
       "9985     0\n",
       "9986     9\n",
       "9987    28\n",
       "9988     1\n",
       "9989    16\n",
       "9990    25\n",
       "9991     9\n",
       "9992     1\n",
       "9993     9\n",
       "9994     1\n",
       "9995    28\n",
       "9996    16\n",
       "9997     0\n",
       "9998    16\n",
       "9999     4\n",
       "Length: 10000, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = pd.Series(np.random.randn(10000) * 10000,dtype = np.int64)\n",
    "print(s1.dtype)\n",
    "# s1.astype(type = np.int64)\n",
    "s1 = s1 * s1 % 36\n",
    "s1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    1\n",
       "2    2\n",
       "3    3\n",
       "4    4\n",
       "5    5\n",
       "6    6\n",
       "7    7\n",
       "8    8\n",
       "9    9\n",
       "dtype: int32"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = np.arange(10)\n",
    "g\n",
    "s2 = pd.Series(g)\n",
    "s2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     A\n",
       "1     B\n",
       "2     C\n",
       "3     D\n",
       "4     E\n",
       "5     F\n",
       "6     G\n",
       "7     H\n",
       "8     I\n",
       "9     J\n",
       "10    K\n",
       "11    L\n",
       "12    M\n",
       "13    N\n",
       "14    O\n",
       "15    P\n",
       "16    Q\n",
       "17    R\n",
       "18    S\n",
       "19    T\n",
       "20    U\n",
       "21    V\n",
       "22    W\n",
       "23    X\n",
       "24    Y\n",
       "25    Z\n",
       "dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array(['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'])\n",
    "a\n",
    "s3 = pd.Series(a)\n",
    "s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     0\n",
       "1     1\n",
       "2     2\n",
       "3     3\n",
       "4     4\n",
       "5     5\n",
       "6     6\n",
       "7     7\n",
       "8     8\n",
       "9     9\n",
       "0     A\n",
       "1     B\n",
       "2     C\n",
       "3     D\n",
       "4     E\n",
       "5     F\n",
       "6     G\n",
       "7     H\n",
       "8     I\n",
       "9     J\n",
       "10    K\n",
       "11    L\n",
       "12    M\n",
       "13    N\n",
       "14    O\n",
       "15    P\n",
       "16    Q\n",
       "17    R\n",
       "18    S\n",
       "19    T\n",
       "20    U\n",
       "21    V\n",
       "22    W\n",
       "23    X\n",
       "24    Y\n",
       "25    Z\n",
       "dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = pd.concat([s2, s3])\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'A', 'B', 'C', 'D', 'E', 'F', 'G',\n",
       "       'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',\n",
       "       'U', 'V', 'W', 'X', 'Y', 'Z'], dtype=object)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'A', 'B', 'C', 'D', 'E', 'F', 'G',\n",
       "       'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',\n",
       "       'U', 'V', 'W', 'X', 'Y', 'Z'], dtype=object)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = s.values\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "36\n",
      "Z\n"
     ]
    }
   ],
   "source": [
    "print(arr[0])\n",
    "print(arr.size)\n",
    "print(arr[35])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  9, 16, ...,  0, 16,  4], dtype=int64)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9970</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9971</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9972</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9973</th>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9974</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9975</th>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9976</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9977</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9978</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9979</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9980</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9981</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9982</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9983</th>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9984</th>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9985</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9986</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9987</th>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9988</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9989</th>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9990</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9991</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9992</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9993</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9994</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     key\n",
       "0      0\n",
       "1      9\n",
       "2      G\n",
       "3      P\n",
       "4      P\n",
       "5      D\n",
       "6      D\n",
       "7      4\n",
       "8      1\n",
       "9      1\n",
       "10     4\n",
       "11     1\n",
       "12     G\n",
       "13     1\n",
       "14     D\n",
       "15     P\n",
       "16     P\n",
       "17     1\n",
       "18     D\n",
       "19     9\n",
       "20     1\n",
       "21     D\n",
       "22     D\n",
       "23     0\n",
       "24     1\n",
       "25     S\n",
       "26     4\n",
       "27     9\n",
       "28     P\n",
       "29     G\n",
       "...   ..\n",
       "9970   1\n",
       "9971   9\n",
       "9972   9\n",
       "9973   G\n",
       "9974   0\n",
       "9975   S\n",
       "9976   P\n",
       "9977   0\n",
       "9978   9\n",
       "9979   9\n",
       "9980   P\n",
       "9981   0\n",
       "9982   D\n",
       "9983   D\n",
       "9984   S\n",
       "9985   0\n",
       "9986   9\n",
       "9987   S\n",
       "9988   1\n",
       "9989   G\n",
       "9990   P\n",
       "9991   9\n",
       "9992   1\n",
       "9993   9\n",
       "9994   1\n",
       "9995   S\n",
       "9996   G\n",
       "9997   0\n",
       "9998   G\n",
       "9999   4\n",
       "\n",
       "[10000 rows x 1 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = pd.DataFrame(arr[s1.values],columns = ['key'])\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "      <th>key</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.204708</td>\n",
       "      <td>0.478943</td>\n",
       "      <td>-0.519439</td>\n",
       "      <td>-0.555730</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.965781</td>\n",
       "      <td>1.393406</td>\n",
       "      <td>0.092908</td>\n",
       "      <td>0.281746</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.769023</td>\n",
       "      <td>1.246435</td>\n",
       "      <td>1.007189</td>\n",
       "      <td>-1.296221</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.274992</td>\n",
       "      <td>0.228913</td>\n",
       "      <td>1.352917</td>\n",
       "      <td>0.886429</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-2.001637</td>\n",
       "      <td>-0.371843</td>\n",
       "      <td>1.669025</td>\n",
       "      <td>-0.438570</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.539741</td>\n",
       "      <td>0.476985</td>\n",
       "      <td>3.248944</td>\n",
       "      <td>-1.021228</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.577087</td>\n",
       "      <td>0.124121</td>\n",
       "      <td>0.302614</td>\n",
       "      <td>0.523772</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000940</td>\n",
       "      <td>1.343810</td>\n",
       "      <td>-0.713544</td>\n",
       "      <td>-0.831154</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-2.370232</td>\n",
       "      <td>-1.860761</td>\n",
       "      <td>-0.860757</td>\n",
       "      <td>0.560145</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-1.265934</td>\n",
       "      <td>0.119827</td>\n",
       "      <td>-1.063512</td>\n",
       "      <td>0.332883</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-2.359419</td>\n",
       "      <td>-0.199543</td>\n",
       "      <td>-1.541996</td>\n",
       "      <td>-0.970736</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-1.307030</td>\n",
       "      <td>0.286350</td>\n",
       "      <td>0.377984</td>\n",
       "      <td>-0.753887</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.331286</td>\n",
       "      <td>1.349742</td>\n",
       "      <td>0.069877</td>\n",
       "      <td>0.246674</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.011862</td>\n",
       "      <td>1.004812</td>\n",
       "      <td>1.327195</td>\n",
       "      <td>-0.919262</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-1.549106</td>\n",
       "      <td>0.022185</td>\n",
       "      <td>0.758363</td>\n",
       "      <td>-0.660524</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.862580</td>\n",
       "      <td>-0.010032</td>\n",
       "      <td>0.050009</td>\n",
       "      <td>0.670216</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.852965</td>\n",
       "      <td>-0.955869</td>\n",
       "      <td>-0.023493</td>\n",
       "      <td>-2.304234</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.652469</td>\n",
       "      <td>-1.218302</td>\n",
       "      <td>-1.332610</td>\n",
       "      <td>1.074623</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.723642</td>\n",
       "      <td>0.690002</td>\n",
       "      <td>1.001543</td>\n",
       "      <td>-0.503087</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-0.622274</td>\n",
       "      <td>-0.921169</td>\n",
       "      <td>-0.726213</td>\n",
       "      <td>0.222896</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.051316</td>\n",
       "      <td>-1.157719</td>\n",
       "      <td>0.816707</td>\n",
       "      <td>0.433610</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.010737</td>\n",
       "      <td>1.824875</td>\n",
       "      <td>-0.997518</td>\n",
       "      <td>0.850591</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-0.131578</td>\n",
       "      <td>0.912414</td>\n",
       "      <td>0.188211</td>\n",
       "      <td>2.169461</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-0.114928</td>\n",
       "      <td>2.003697</td>\n",
       "      <td>0.029610</td>\n",
       "      <td>0.795253</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.118110</td>\n",
       "      <td>-0.748532</td>\n",
       "      <td>0.584970</td>\n",
       "      <td>0.152677</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-1.565657</td>\n",
       "      <td>-0.562540</td>\n",
       "      <td>-0.032664</td>\n",
       "      <td>-0.929006</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-0.482573</td>\n",
       "      <td>-0.036264</td>\n",
       "      <td>1.095390</td>\n",
       "      <td>0.980928</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.589488</td>\n",
       "      <td>1.581700</td>\n",
       "      <td>-0.528735</td>\n",
       "      <td>0.457002</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.929969</td>\n",
       "      <td>-1.569271</td>\n",
       "      <td>-1.022487</td>\n",
       "      <td>-0.402827</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.220487</td>\n",
       "      <td>-0.193401</td>\n",
       "      <td>0.669158</td>\n",
       "      <td>-1.648985</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9970</th>\n",
       "      <td>-0.515691</td>\n",
       "      <td>1.244901</td>\n",
       "      <td>1.302043</td>\n",
       "      <td>-1.034539</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9971</th>\n",
       "      <td>-0.597495</td>\n",
       "      <td>1.385267</td>\n",
       "      <td>-0.763196</td>\n",
       "      <td>1.990051</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9972</th>\n",
       "      <td>1.914450</td>\n",
       "      <td>0.382984</td>\n",
       "      <td>0.809112</td>\n",
       "      <td>-1.745766</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9973</th>\n",
       "      <td>-1.281518</td>\n",
       "      <td>0.008116</td>\n",
       "      <td>-1.212094</td>\n",
       "      <td>-0.635271</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9974</th>\n",
       "      <td>-1.141587</td>\n",
       "      <td>2.319184</td>\n",
       "      <td>-1.294746</td>\n",
       "      <td>-0.252059</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9975</th>\n",
       "      <td>0.739082</td>\n",
       "      <td>0.966051</td>\n",
       "      <td>-1.178064</td>\n",
       "      <td>0.535442</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9976</th>\n",
       "      <td>-0.051060</td>\n",
       "      <td>-0.312800</td>\n",
       "      <td>1.618086</td>\n",
       "      <td>2.258698</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9977</th>\n",
       "      <td>-1.452134</td>\n",
       "      <td>1.808934</td>\n",
       "      <td>2.480793</td>\n",
       "      <td>0.926589</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9978</th>\n",
       "      <td>-1.453201</td>\n",
       "      <td>0.369264</td>\n",
       "      <td>1.018868</td>\n",
       "      <td>-1.408517</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9979</th>\n",
       "      <td>0.060708</td>\n",
       "      <td>-1.048864</td>\n",
       "      <td>-0.611300</td>\n",
       "      <td>0.372017</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9980</th>\n",
       "      <td>1.301401</td>\n",
       "      <td>-1.171101</td>\n",
       "      <td>-1.461861</td>\n",
       "      <td>-0.269016</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9981</th>\n",
       "      <td>1.124364</td>\n",
       "      <td>-0.900691</td>\n",
       "      <td>-0.949605</td>\n",
       "      <td>-0.845981</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9982</th>\n",
       "      <td>-2.051208</td>\n",
       "      <td>0.775944</td>\n",
       "      <td>0.985752</td>\n",
       "      <td>-1.302367</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9983</th>\n",
       "      <td>-0.739562</td>\n",
       "      <td>0.875037</td>\n",
       "      <td>-0.521750</td>\n",
       "      <td>0.275383</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9984</th>\n",
       "      <td>-0.323421</td>\n",
       "      <td>-0.832905</td>\n",
       "      <td>-1.423236</td>\n",
       "      <td>0.957248</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9985</th>\n",
       "      <td>1.586598</td>\n",
       "      <td>-1.465278</td>\n",
       "      <td>-1.184578</td>\n",
       "      <td>0.515831</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9986</th>\n",
       "      <td>-0.110349</td>\n",
       "      <td>0.532887</td>\n",
       "      <td>0.332079</td>\n",
       "      <td>0.349169</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9987</th>\n",
       "      <td>0.278290</td>\n",
       "      <td>-0.860830</td>\n",
       "      <td>0.505153</td>\n",
       "      <td>0.817113</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9988</th>\n",
       "      <td>-0.434324</td>\n",
       "      <td>-1.897628</td>\n",
       "      <td>0.565004</td>\n",
       "      <td>1.508078</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9989</th>\n",
       "      <td>-0.370955</td>\n",
       "      <td>-0.777062</td>\n",
       "      <td>-1.072253</td>\n",
       "      <td>-0.007181</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9990</th>\n",
       "      <td>1.000089</td>\n",
       "      <td>1.167074</td>\n",
       "      <td>-0.188872</td>\n",
       "      <td>1.099893</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9991</th>\n",
       "      <td>0.202083</td>\n",
       "      <td>0.626895</td>\n",
       "      <td>-0.562755</td>\n",
       "      <td>0.851409</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9992</th>\n",
       "      <td>0.947399</td>\n",
       "      <td>0.137731</td>\n",
       "      <td>-1.341261</td>\n",
       "      <td>1.594911</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9993</th>\n",
       "      <td>0.241749</td>\n",
       "      <td>0.613748</td>\n",
       "      <td>0.270834</td>\n",
       "      <td>0.915012</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9994</th>\n",
       "      <td>2.825056</td>\n",
       "      <td>1.785693</td>\n",
       "      <td>-0.068129</td>\n",
       "      <td>-1.662357</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0.481378</td>\n",
       "      <td>-0.735848</td>\n",
       "      <td>0.426780</td>\n",
       "      <td>-0.616316</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>0.400465</td>\n",
       "      <td>-0.173379</td>\n",
       "      <td>-0.784637</td>\n",
       "      <td>0.326710</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>-0.847900</td>\n",
       "      <td>-0.462586</td>\n",
       "      <td>-0.574423</td>\n",
       "      <td>0.632782</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>1.299831</td>\n",
       "      <td>1.760948</td>\n",
       "      <td>2.089599</td>\n",
       "      <td>0.091444</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>0.061865</td>\n",
       "      <td>0.781684</td>\n",
       "      <td>-0.045915</td>\n",
       "      <td>-0.773583</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           one       two     three      four key\n",
       "0    -0.204708  0.478943 -0.519439 -0.555730   0\n",
       "1     1.965781  1.393406  0.092908  0.281746   9\n",
       "2     0.769023  1.246435  1.007189 -1.296221   G\n",
       "3     0.274992  0.228913  1.352917  0.886429   P\n",
       "4    -2.001637 -0.371843  1.669025 -0.438570   P\n",
       "5    -0.539741  0.476985  3.248944 -1.021228   D\n",
       "6    -0.577087  0.124121  0.302614  0.523772   D\n",
       "7     0.000940  1.343810 -0.713544 -0.831154   4\n",
       "8    -2.370232 -1.860761 -0.860757  0.560145   1\n",
       "9    -1.265934  0.119827 -1.063512  0.332883   1\n",
       "10   -2.359419 -0.199543 -1.541996 -0.970736   4\n",
       "11   -1.307030  0.286350  0.377984 -0.753887   1\n",
       "12    0.331286  1.349742  0.069877  0.246674   G\n",
       "13   -0.011862  1.004812  1.327195 -0.919262   1\n",
       "14   -1.549106  0.022185  0.758363 -0.660524   D\n",
       "15    0.862580 -0.010032  0.050009  0.670216   P\n",
       "16    0.852965 -0.955869 -0.023493 -2.304234   P\n",
       "17   -0.652469 -1.218302 -1.332610  1.074623   1\n",
       "18    0.723642  0.690002  1.001543 -0.503087   D\n",
       "19   -0.622274 -0.921169 -0.726213  0.222896   9\n",
       "20    0.051316 -1.157719  0.816707  0.433610   1\n",
       "21    1.010737  1.824875 -0.997518  0.850591   D\n",
       "22   -0.131578  0.912414  0.188211  2.169461   D\n",
       "23   -0.114928  2.003697  0.029610  0.795253   0\n",
       "24    0.118110 -0.748532  0.584970  0.152677   1\n",
       "25   -1.565657 -0.562540 -0.032664 -0.929006   S\n",
       "26   -0.482573 -0.036264  1.095390  0.980928   4\n",
       "27   -0.589488  1.581700 -0.528735  0.457002   9\n",
       "28    0.929969 -1.569271 -1.022487 -0.402827   P\n",
       "29    0.220487 -0.193401  0.669158 -1.648985   G\n",
       "...        ...       ...       ...       ...  ..\n",
       "9970 -0.515691  1.244901  1.302043 -1.034539   1\n",
       "9971 -0.597495  1.385267 -0.763196  1.990051   9\n",
       "9972  1.914450  0.382984  0.809112 -1.745766   9\n",
       "9973 -1.281518  0.008116 -1.212094 -0.635271   G\n",
       "9974 -1.141587  2.319184 -1.294746 -0.252059   0\n",
       "9975  0.739082  0.966051 -1.178064  0.535442   S\n",
       "9976 -0.051060 -0.312800  1.618086  2.258698   P\n",
       "9977 -1.452134  1.808934  2.480793  0.926589   0\n",
       "9978 -1.453201  0.369264  1.018868 -1.408517   9\n",
       "9979  0.060708 -1.048864 -0.611300  0.372017   9\n",
       "9980  1.301401 -1.171101 -1.461861 -0.269016   P\n",
       "9981  1.124364 -0.900691 -0.949605 -0.845981   0\n",
       "9982 -2.051208  0.775944  0.985752 -1.302367   D\n",
       "9983 -0.739562  0.875037 -0.521750  0.275383   D\n",
       "9984 -0.323421 -0.832905 -1.423236  0.957248   S\n",
       "9985  1.586598 -1.465278 -1.184578  0.515831   0\n",
       "9986 -0.110349  0.532887  0.332079  0.349169   9\n",
       "9987  0.278290 -0.860830  0.505153  0.817113   S\n",
       "9988 -0.434324 -1.897628  0.565004  1.508078   1\n",
       "9989 -0.370955 -0.777062 -1.072253 -0.007181   G\n",
       "9990  1.000089  1.167074 -0.188872  1.099893   P\n",
       "9991  0.202083  0.626895 -0.562755  0.851409   9\n",
       "9992  0.947399  0.137731 -1.341261  1.594911   1\n",
       "9993  0.241749  0.613748  0.270834  0.915012   9\n",
       "9994  2.825056  1.785693 -0.068129 -1.662357   1\n",
       "9995  0.481378 -0.735848  0.426780 -0.616316   S\n",
       "9996  0.400465 -0.173379 -0.784637  0.326710   G\n",
       "9997 -0.847900 -0.462586 -0.574423  0.632782   0\n",
       "9998  1.299831  1.760948  2.089599  0.091444   G\n",
       "9999  0.061865  0.781684 -0.045915 -0.773583   4\n",
       "\n",
       "[10000 rows x 5 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3 = pd.concat([df1, df2] ,axis = 1)\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3.to_csv('./4/data/data6.csv',index=None) #index = None 避免导出时带有索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "      <th>key</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.204708</td>\n",
       "      <td>0.478943</td>\n",
       "      <td>-0.519439</td>\n",
       "      <td>-0.555730</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.965781</td>\n",
       "      <td>1.393406</td>\n",
       "      <td>0.092908</td>\n",
       "      <td>0.281746</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.769023</td>\n",
       "      <td>1.246435</td>\n",
       "      <td>1.007189</td>\n",
       "      <td>-1.296221</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.274992</td>\n",
       "      <td>0.228913</td>\n",
       "      <td>1.352917</td>\n",
       "      <td>0.886429</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-2.001637</td>\n",
       "      <td>-0.371843</td>\n",
       "      <td>1.669025</td>\n",
       "      <td>-0.438570</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.539741</td>\n",
       "      <td>0.476985</td>\n",
       "      <td>3.248944</td>\n",
       "      <td>-1.021228</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.577087</td>\n",
       "      <td>0.124121</td>\n",
       "      <td>0.302614</td>\n",
       "      <td>0.523772</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000940</td>\n",
       "      <td>1.343810</td>\n",
       "      <td>-0.713544</td>\n",
       "      <td>-0.831154</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-2.370232</td>\n",
       "      <td>-1.860761</td>\n",
       "      <td>-0.860757</td>\n",
       "      <td>0.560145</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-1.265934</td>\n",
       "      <td>0.119827</td>\n",
       "      <td>-1.063512</td>\n",
       "      <td>0.332883</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-2.359419</td>\n",
       "      <td>-0.199543</td>\n",
       "      <td>-1.541996</td>\n",
       "      <td>-0.970736</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-1.307030</td>\n",
       "      <td>0.286350</td>\n",
       "      <td>0.377984</td>\n",
       "      <td>-0.753887</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.331286</td>\n",
       "      <td>1.349742</td>\n",
       "      <td>0.069877</td>\n",
       "      <td>0.246674</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.011862</td>\n",
       "      <td>1.004812</td>\n",
       "      <td>1.327195</td>\n",
       "      <td>-0.919262</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-1.549106</td>\n",
       "      <td>0.022185</td>\n",
       "      <td>0.758363</td>\n",
       "      <td>-0.660524</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.862580</td>\n",
       "      <td>-0.010032</td>\n",
       "      <td>0.050009</td>\n",
       "      <td>0.670216</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.852965</td>\n",
       "      <td>-0.955869</td>\n",
       "      <td>-0.023493</td>\n",
       "      <td>-2.304234</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.652469</td>\n",
       "      <td>-1.218302</td>\n",
       "      <td>-1.332610</td>\n",
       "      <td>1.074623</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.723642</td>\n",
       "      <td>0.690002</td>\n",
       "      <td>1.001543</td>\n",
       "      <td>-0.503087</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-0.622274</td>\n",
       "      <td>-0.921169</td>\n",
       "      <td>-0.726213</td>\n",
       "      <td>0.222896</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.051316</td>\n",
       "      <td>-1.157719</td>\n",
       "      <td>0.816707</td>\n",
       "      <td>0.433610</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.010737</td>\n",
       "      <td>1.824875</td>\n",
       "      <td>-0.997518</td>\n",
       "      <td>0.850591</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-0.131578</td>\n",
       "      <td>0.912414</td>\n",
       "      <td>0.188211</td>\n",
       "      <td>2.169461</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-0.114928</td>\n",
       "      <td>2.003697</td>\n",
       "      <td>0.029610</td>\n",
       "      <td>0.795253</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.118110</td>\n",
       "      <td>-0.748532</td>\n",
       "      <td>0.584970</td>\n",
       "      <td>0.152677</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-1.565657</td>\n",
       "      <td>-0.562540</td>\n",
       "      <td>-0.032664</td>\n",
       "      <td>-0.929006</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-0.482573</td>\n",
       "      <td>-0.036264</td>\n",
       "      <td>1.095390</td>\n",
       "      <td>0.980928</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.589488</td>\n",
       "      <td>1.581700</td>\n",
       "      <td>-0.528735</td>\n",
       "      <td>0.457002</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.929969</td>\n",
       "      <td>-1.569271</td>\n",
       "      <td>-1.022487</td>\n",
       "      <td>-0.402827</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.220487</td>\n",
       "      <td>-0.193401</td>\n",
       "      <td>0.669158</td>\n",
       "      <td>-1.648985</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9970</th>\n",
       "      <td>-0.515691</td>\n",
       "      <td>1.244901</td>\n",
       "      <td>1.302043</td>\n",
       "      <td>-1.034539</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9971</th>\n",
       "      <td>-0.597495</td>\n",
       "      <td>1.385267</td>\n",
       "      <td>-0.763196</td>\n",
       "      <td>1.990051</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9972</th>\n",
       "      <td>1.914450</td>\n",
       "      <td>0.382984</td>\n",
       "      <td>0.809112</td>\n",
       "      <td>-1.745766</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9973</th>\n",
       "      <td>-1.281518</td>\n",
       "      <td>0.008116</td>\n",
       "      <td>-1.212094</td>\n",
       "      <td>-0.635271</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9974</th>\n",
       "      <td>-1.141587</td>\n",
       "      <td>2.319184</td>\n",
       "      <td>-1.294746</td>\n",
       "      <td>-0.252059</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9975</th>\n",
       "      <td>0.739082</td>\n",
       "      <td>0.966051</td>\n",
       "      <td>-1.178064</td>\n",
       "      <td>0.535442</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9976</th>\n",
       "      <td>-0.051060</td>\n",
       "      <td>-0.312800</td>\n",
       "      <td>1.618086</td>\n",
       "      <td>2.258698</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9977</th>\n",
       "      <td>-1.452134</td>\n",
       "      <td>1.808934</td>\n",
       "      <td>2.480793</td>\n",
       "      <td>0.926589</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9978</th>\n",
       "      <td>-1.453201</td>\n",
       "      <td>0.369264</td>\n",
       "      <td>1.018868</td>\n",
       "      <td>-1.408517</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9979</th>\n",
       "      <td>0.060708</td>\n",
       "      <td>-1.048864</td>\n",
       "      <td>-0.611300</td>\n",
       "      <td>0.372017</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9980</th>\n",
       "      <td>1.301401</td>\n",
       "      <td>-1.171101</td>\n",
       "      <td>-1.461861</td>\n",
       "      <td>-0.269016</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9981</th>\n",
       "      <td>1.124364</td>\n",
       "      <td>-0.900691</td>\n",
       "      <td>-0.949605</td>\n",
       "      <td>-0.845981</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9982</th>\n",
       "      <td>-2.051208</td>\n",
       "      <td>0.775944</td>\n",
       "      <td>0.985752</td>\n",
       "      <td>-1.302367</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9983</th>\n",
       "      <td>-0.739562</td>\n",
       "      <td>0.875037</td>\n",
       "      <td>-0.521750</td>\n",
       "      <td>0.275383</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9984</th>\n",
       "      <td>-0.323421</td>\n",
       "      <td>-0.832905</td>\n",
       "      <td>-1.423236</td>\n",
       "      <td>0.957248</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9985</th>\n",
       "      <td>1.586598</td>\n",
       "      <td>-1.465278</td>\n",
       "      <td>-1.184578</td>\n",
       "      <td>0.515831</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9986</th>\n",
       "      <td>-0.110349</td>\n",
       "      <td>0.532887</td>\n",
       "      <td>0.332079</td>\n",
       "      <td>0.349169</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9987</th>\n",
       "      <td>0.278290</td>\n",
       "      <td>-0.860830</td>\n",
       "      <td>0.505153</td>\n",
       "      <td>0.817113</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9988</th>\n",
       "      <td>-0.434324</td>\n",
       "      <td>-1.897628</td>\n",
       "      <td>0.565004</td>\n",
       "      <td>1.508078</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9989</th>\n",
       "      <td>-0.370955</td>\n",
       "      <td>-0.777062</td>\n",
       "      <td>-1.072253</td>\n",
       "      <td>-0.007181</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9990</th>\n",
       "      <td>1.000089</td>\n",
       "      <td>1.167074</td>\n",
       "      <td>-0.188872</td>\n",
       "      <td>1.099893</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9991</th>\n",
       "      <td>0.202083</td>\n",
       "      <td>0.626895</td>\n",
       "      <td>-0.562755</td>\n",
       "      <td>0.851409</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9992</th>\n",
       "      <td>0.947399</td>\n",
       "      <td>0.137731</td>\n",
       "      <td>-1.341261</td>\n",
       "      <td>1.594911</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9993</th>\n",
       "      <td>0.241749</td>\n",
       "      <td>0.613748</td>\n",
       "      <td>0.270834</td>\n",
       "      <td>0.915012</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9994</th>\n",
       "      <td>2.825056</td>\n",
       "      <td>1.785693</td>\n",
       "      <td>-0.068129</td>\n",
       "      <td>-1.662357</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0.481378</td>\n",
       "      <td>-0.735848</td>\n",
       "      <td>0.426780</td>\n",
       "      <td>-0.616316</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>0.400465</td>\n",
       "      <td>-0.173379</td>\n",
       "      <td>-0.784637</td>\n",
       "      <td>0.326710</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>-0.847900</td>\n",
       "      <td>-0.462586</td>\n",
       "      <td>-0.574423</td>\n",
       "      <td>0.632782</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>1.299831</td>\n",
       "      <td>1.760948</td>\n",
       "      <td>2.089599</td>\n",
       "      <td>0.091444</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>0.061865</td>\n",
       "      <td>0.781684</td>\n",
       "      <td>-0.045915</td>\n",
       "      <td>-0.773583</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           one       two     three      four key\n",
       "0    -0.204708  0.478943 -0.519439 -0.555730   0\n",
       "1     1.965781  1.393406  0.092908  0.281746   9\n",
       "2     0.769023  1.246435  1.007189 -1.296221   G\n",
       "3     0.274992  0.228913  1.352917  0.886429   P\n",
       "4    -2.001637 -0.371843  1.669025 -0.438570   P\n",
       "5    -0.539741  0.476985  3.248944 -1.021228   D\n",
       "6    -0.577087  0.124121  0.302614  0.523772   D\n",
       "7     0.000940  1.343810 -0.713544 -0.831154   4\n",
       "8    -2.370232 -1.860761 -0.860757  0.560145   1\n",
       "9    -1.265934  0.119827 -1.063512  0.332883   1\n",
       "10   -2.359419 -0.199543 -1.541996 -0.970736   4\n",
       "11   -1.307030  0.286350  0.377984 -0.753887   1\n",
       "12    0.331286  1.349742  0.069877  0.246674   G\n",
       "13   -0.011862  1.004812  1.327195 -0.919262   1\n",
       "14   -1.549106  0.022185  0.758363 -0.660524   D\n",
       "15    0.862580 -0.010032  0.050009  0.670216   P\n",
       "16    0.852965 -0.955869 -0.023493 -2.304234   P\n",
       "17   -0.652469 -1.218302 -1.332610  1.074623   1\n",
       "18    0.723642  0.690002  1.001543 -0.503087   D\n",
       "19   -0.622274 -0.921169 -0.726213  0.222896   9\n",
       "20    0.051316 -1.157719  0.816707  0.433610   1\n",
       "21    1.010737  1.824875 -0.997518  0.850591   D\n",
       "22   -0.131578  0.912414  0.188211  2.169461   D\n",
       "23   -0.114928  2.003697  0.029610  0.795253   0\n",
       "24    0.118110 -0.748532  0.584970  0.152677   1\n",
       "25   -1.565657 -0.562540 -0.032664 -0.929006   S\n",
       "26   -0.482573 -0.036264  1.095390  0.980928   4\n",
       "27   -0.589488  1.581700 -0.528735  0.457002   9\n",
       "28    0.929969 -1.569271 -1.022487 -0.402827   P\n",
       "29    0.220487 -0.193401  0.669158 -1.648985   G\n",
       "...        ...       ...       ...       ...  ..\n",
       "9970 -0.515691  1.244901  1.302043 -1.034539   1\n",
       "9971 -0.597495  1.385267 -0.763196  1.990051   9\n",
       "9972  1.914450  0.382984  0.809112 -1.745766   9\n",
       "9973 -1.281518  0.008116 -1.212094 -0.635271   G\n",
       "9974 -1.141587  2.319184 -1.294746 -0.252059   0\n",
       "9975  0.739082  0.966051 -1.178064  0.535442   S\n",
       "9976 -0.051060 -0.312800  1.618086  2.258698   P\n",
       "9977 -1.452134  1.808934  2.480793  0.926589   0\n",
       "9978 -1.453201  0.369264  1.018868 -1.408517   9\n",
       "9979  0.060708 -1.048864 -0.611300  0.372017   9\n",
       "9980  1.301401 -1.171101 -1.461861 -0.269016   P\n",
       "9981  1.124364 -0.900691 -0.949605 -0.845981   0\n",
       "9982 -2.051208  0.775944  0.985752 -1.302367   D\n",
       "9983 -0.739562  0.875037 -0.521750  0.275383   D\n",
       "9984 -0.323421 -0.832905 -1.423236  0.957248   S\n",
       "9985  1.586598 -1.465278 -1.184578  0.515831   0\n",
       "9986 -0.110349  0.532887  0.332079  0.349169   9\n",
       "9987  0.278290 -0.860830  0.505153  0.817113   S\n",
       "9988 -0.434324 -1.897628  0.565004  1.508078   1\n",
       "9989 -0.370955 -0.777062 -1.072253 -0.007181   G\n",
       "9990  1.000089  1.167074 -0.188872  1.099893   P\n",
       "9991  0.202083  0.626895 -0.562755  0.851409   9\n",
       "9992  0.947399  0.137731 -1.341261  1.594911   1\n",
       "9993  0.241749  0.613748  0.270834  0.915012   9\n",
       "9994  2.825056  1.785693 -0.068129 -1.662357   1\n",
       "9995  0.481378 -0.735848  0.426780 -0.616316   S\n",
       "9996  0.400465 -0.173379 -0.784637  0.326710   G\n",
       "9997 -0.847900 -0.462586 -0.574423  0.632782   0\n",
       "9998  1.299831  1.760948  2.089599  0.091444   G\n",
       "9999  0.061865  0.781684 -0.045915 -0.773583   4\n",
       "\n",
       "[10000 rows x 5 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.read_csv('./4/data/data6.csv')\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "      <th>key</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.204708</td>\n",
       "      <td>0.478943</td>\n",
       "      <td>-0.519439</td>\n",
       "      <td>-0.555730</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.965781</td>\n",
       "      <td>1.393406</td>\n",
       "      <td>0.092908</td>\n",
       "      <td>0.281746</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.769023</td>\n",
       "      <td>1.246435</td>\n",
       "      <td>1.007189</td>\n",
       "      <td>-1.296221</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.274992</td>\n",
       "      <td>0.228913</td>\n",
       "      <td>1.352917</td>\n",
       "      <td>0.886429</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-2.001637</td>\n",
       "      <td>-0.371843</td>\n",
       "      <td>1.669025</td>\n",
       "      <td>-0.438570</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        one       two     three      four key\n",
       "0 -0.204708  0.478943 -0.519439 -0.555730   0\n",
       "1  1.965781  1.393406  0.092908  0.281746   9\n",
       "2  0.769023  1.246435  1.007189 -1.296221   G\n",
       "3  0.274992  0.228913  1.352917  0.886429   P\n",
       "4 -2.001637 -0.371843  1.669025 -0.438570   P"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('./4/data/data6.csv', nrows = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.io.parsers.TextFileReader at 0x9b09470>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chunker = pd.read_csv('./4/data/data6.csv', chunksize = 100)\n",
    "chunker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunker = pd.read_csv('./4/data/data6.csv', chunksize = 100)\n",
    "\n",
    "tot = Series([])\n",
    "for piece in chunker:\n",
    "    tot = tot.add(piece['key'].value_counts(), fill_value = 0)\n",
    "    \n",
    "# tot = tot.order(ascending = False) # order 已经过时了\n",
    "tot = tot.sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1704.0\n",
       "9    1619.0\n",
       "S    1165.0\n",
       "D    1136.0\n",
       "G    1117.0\n",
       "P    1109.0\n",
       "1    1095.0\n",
       "4    1055.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tot[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 把数据写入文本格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>something</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>three</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  something  a   b     c   d message\n",
       "0       one  1   2   3.0   4     NaN\n",
       "1       two  5   6   NaN   8   world\n",
       "2     three  9  10  11.0  12     foo"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('./4/data/data5.csv')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>something</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>three</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  something  a   b     c   d message\n",
       "0       one  1   2   3.0   4     NaN\n",
       "1       two  5   6   NaN   8   world\n",
       "2     three  9  10  11.0  12     foo"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.to_csv('./4/data/out.csv',index=None)\n",
    "# !cat ./4/data/out.csv\n",
    "\n",
    "pd.read_csv('./4/data/out.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|something|a|b|c|d|message\n",
      "0|one|1|2|3.0|4|\n",
      "1|two|5|6||8|world\n",
      "2|three|9|10|11.0|12|foo\n"
     ]
    }
   ],
   "source": [
    "data.to_csv(sys.stdout, sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ",something,a,b,c,d,message\n",
      "0,one,1,2,3.0,4,NULL\n",
      "1,two,5,6,NULL,8,world\n",
      "2,three,9,10,11.0,12,foo\n"
     ]
    }
   ],
   "source": [
    "data.to_csv(sys.stdout, na_rep = 'NULL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "one,1,2,3.0,4,\n",
      "two,5,6,,8,world\n",
      "three,9,10,11.0,12,foo\n"
     ]
    }
   ],
   "source": [
    "data.to_csv(sys.stdout, index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a,b,c\n",
      "1,2,3.0\n",
      "5,6,\n",
      "9,10,11.0\n"
     ]
    }
   ],
   "source": [
    "data.to_csv(sys.stdout, index=False, columns=['a', 'b', 'c'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000-01-01,0\n",
      "2000-01-02,1\n",
      "2000-01-03,2\n",
      "2000-01-04,3\n",
      "2000-01-05,4\n",
      "2000-01-06,5\n",
      "2000-01-07,6\n"
     ]
    }
   ],
   "source": [
    "dates = pd.date_range('1/1/2000', periods=7)\n",
    "ts = Series(np.arange(7), index=dates)\n",
    "ts.to_csv('./4/data/tseries.csv')\n",
    "# !cat ./4/data/tseries.csv # 不能使用 cat 展示 csv 数据 jiu\n",
    "tseries = pd.read_csv('./4/data/tseries.csv')\n",
    "tseries.to_csv(sys.stdout,index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.4 手动读写数据（按要求）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"a\",\"b\",\"c\"\n",
      "\"1\",\"2\",\"3\"\n",
      "\"1\",\"2\",\"3\",\"4\"\n"
     ]
    }
   ],
   "source": [
    "# r 模式打开并读取\n",
    "fp = open('./4/data/data7.csv',\"r\")\n",
    "content = fp.read()\n",
    "fp.close()\n",
    "print(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "f = open('./4/data/data7.csv')\n",
    "\n",
    "reader = csv.reader(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a', 'b', 'c']\n",
      "['1', '2', '3']\n",
      "['1', '2', '3', '4']\n"
     ]
    }
   ],
   "source": [
    "for line in reader:\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"a\",\"b\",\"c\"\r\n",
      "\"1\",\"2\",\"3\"\r\n",
      "\"1\",\"2\",\"3\",\"4\"\n"
     ]
    }
   ],
   "source": [
    "# rb 模式打开并读取 用于读取二进制文件\n",
    "fp = open('./4/data/data7.csv',\"rb\")\n",
    "content = fp.read()\n",
    "fp.close()\n",
    "print(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"a\",\"b\",\"c\"\\r\\n', '\"1\",\"2\",\"3\"\\r\\n', '\"1\",\"2\",\"3\",\"4\"']\n"
     ]
    }
   ],
   "source": [
    "# 使用 readline 方法 读取\n",
    "list_r = []\n",
    "fp= open('./4/data/data7.csv',\"rb\")\n",
    "done = False;\n",
    "while not done:\n",
    "    aline = fp.readline();\n",
    "    if(aline != \"\"):\n",
    "        list_r.append(aline)\n",
    "    else:\n",
    "        done = True\n",
    "fp.close()\n",
    "print(list_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"a\",\"b\",\"c\"\\n', '\"1\",\"2\",\"3\"\\n', '\"1\",\"2\",\"3\",\"4\"']\n"
     ]
    }
   ],
   "source": [
    "## 使用 readlines 方法 读取\n",
    "list_r = []\n",
    "fp = open('./4/data/data7.csv',\"r\")\n",
    "for line in fp.readlines():\n",
    "    list_r.append(line)\n",
    "fp.close()\n",
    "print(list_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"a\",\"b\",\"c\"\n",
      "\"1\",\"2\",\"3\"\n",
      "\"1\",\"2\",\"3\",\"4\"\n"
     ]
    }
   ],
   "source": [
    "print open('./4/data/data7.csv','r').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a': ('1', '1'), 'b': ('2', '2'), 'c': ('3', '3')}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = list(csv.reader(open('./4/data/data7.csv')))\n",
    "header, values = lines[0], lines[1:]\n",
    "data_dict = {h: v for h, v in zip(header, zip(*values))}\n",
    "data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "class my_dialect(csv.Dialect):\n",
    "    lineterminator = '\\n'\n",
    "    delimiter = ';'\n",
    "    quotechar = '\"'\n",
    "    quoting = csv.QUOTE_MINIMAL\n",
    "with open('./4/data/mydata.csv', 'w') as f:\n",
    "    writer = csv.writer(f, dialect=my_dialect)\n",
    "    writer.writerow(('one', 'two', 'three'))\n",
    "    writer.writerow(('1', '2', '3'))\n",
    "    writer.writerow(('4', '5', '6'))\n",
    "    writer.writerow(('7', '8', '9'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "one;two;three\n",
      "1;2;3\n",
      "4;5;6\n",
      "7;8;9\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print( open('./4/data/mydata.csv').read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.5 JSON格式的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "obj = \\\n",
    "\"\"\"\n",
    "{\"姓名\":\"张三\",\n",
    "\"住处\":[\"天朝\",\"挖每个\",\"万恶的资本主义日不落帝国\"],\n",
    "\"宠物\":null,\n",
    "\"兄弟\":[{\"姓名\":\"李四\",\"年龄\": 25, \"宠物\": \"汪星人\"},\n",
    "            {\"姓名\":\"王五\",\"年龄\": 23, \"宠物\": \"喵星人\"}]\n",
    "}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'\\u4f4f\\u5904': [u'\\u5929\\u671d',\n",
       "  u'\\u6316\\u6bcf\\u4e2a',\n",
       "  u'\\u4e07\\u6076\\u7684\\u8d44\\u672c\\u4e3b\\u4e49\\u65e5\\u4e0d\\u843d\\u5e1d\\u56fd'],\n",
       " u'\\u5144\\u5f1f': [{u'\\u59d3\\u540d': u'\\u674e\\u56db',\n",
       "   u'\\u5ba0\\u7269': u'\\u6c6a\\u661f\\u4eba',\n",
       "   u'\\u5e74\\u9f84': 25},\n",
       "  {u'\\u59d3\\u540d': u'\\u738b\\u4e94',\n",
       "   u'\\u5ba0\\u7269': u'\\u55b5\\u661f\\u4eba',\n",
       "   u'\\u5e74\\u9f84': 23}],\n",
       " u'\\u59d3\\u540d': u'\\u5f20\\u4e09',\n",
       " u'\\u5ba0\\u7269': None}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "result = json.loads(obj)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"兄弟\": [{\"年龄\": 25, \"宠物\": \"汪星人\", \"姓名\": \"李四\"}, {\"年龄\": 23, \"宠物\": \"喵星人\", \"姓名\": \"王五\"}], \"住处\": [\"天朝\", \"挖每个\", \"万恶的资本主义日不落帝国\"], \"宠物\": null, \"姓名\": \"张三\"}\n"
     ]
    }
   ],
   "source": [
    "print json.dumps(result, encoding=\"UTF-8\", ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{u'\\u59d3\\u540d': u'\\u674e\\u56db',\n",
       "  u'\\u5ba0\\u7269': u'\\u6c6a\\u661f\\u4eba',\n",
       "  u'\\u5e74\\u9f84': 25},\n",
       " {u'\\u59d3\\u540d': u'\\u738b\\u4e94',\n",
       "  u'\\u5ba0\\u7269': u'\\u55b5\\u661f\\u4eba',\n",
       "  u'\\u5e74\\u9f84': 23}]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[u'兄弟']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"年龄\": 25, \"宠物\": \"汪星人\", \"姓名\": \"李四\"}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(result[u\"兄弟\"][0],encoding=\"UTF-8\", ensure_ascii=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "asjson = json.dumps(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>姓名</th>\n",
       "      <th>年龄</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>李四</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>王五</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   姓名  年龄\n",
       "0  李四  25\n",
       "1  王五  23"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brothers = DataFrame(result[u'兄弟'], columns=[u'姓名',u'年龄'])\n",
    "brothers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.6 人人都爱爬虫，人人都要解析XML 和 HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml.html import parse\n",
    "from urllib2 import urlopen\n",
    "\n",
    "parsed = parse(urlopen('https://ask.julyedu.com/'))\n",
    "\n",
    "doc = parsed.getroot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Element html at 0x9df5728>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Element a at 0x9f671d8>,\n",
       " <Element a at 0x9f67228>,\n",
       " <Element a at 0x9f67278>,\n",
       " <Element a at 0x9f672c8>,\n",
       " <Element a at 0x9f67318>]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links = doc.findall('.//a') #双斜杠表示在全局寻找a 不在乎在那一层\n",
    "links[15:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                    精品课程\n"
     ]
    }
   ],
   "source": [
    "lnk = links[14]\n",
    "lnk\n",
    "lnk.get('href')\n",
    "print lnk.text_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://weibo.com/askjulyedu',\n",
       " None,\n",
       " 'https://www.julyedu.com/help/index/about',\n",
       " 'https://www.julyedu.com/help/index/contact',\n",
       " 'https://www.julyedu.com/help/index/join',\n",
       " 'https://ask.julyedu.com/question/55',\n",
       " 'http://www.miitbeian.gov.cn/',\n",
       " 'https://tianchi.aliyun.com',\n",
       " 'https://cloud.tencent.com/developer/edu',\n",
       " 'https://www.aidaxue.com/?ch=qyzx']"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls = [lnk.get('href') for lnk in doc.findall('.//a')]\n",
    "urls[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "129"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spans = doc.findall('.//span')\n",
    "len(spans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NEW\n",
      "赚奖学金\n",
      "购物车\n",
      "\n",
      " 购物车\n",
      " 通知设置\n",
      "贡献\n",
      "回复了问题 • 8 人关注 • 7 个回复 • 877 次浏览 • 1 天前\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 21 次浏览 • 1 天前 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 4 人关注 • 4 个回复 • 1410 次浏览 • 2 天前\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 5 人关注 • 4 个回复 • 1461 次浏览 • 5 天前\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 49 次浏览 • 5 天前 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 48 次浏览 • 5 天前 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 49 次浏览 • 2019-03-13 14:49 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 15 人关注 • 12 个回复 • 1714 次浏览 • 2019-03-12 18:09\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 6 人关注 • 4 个回复 • 11399 次浏览 • 2019-03-07 23:49\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 785 次浏览 • 2019-03-06 14:30 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 5 人关注 • 2 个回复 • 1641 次浏览 • 2019-03-06 11:55\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 3 人关注 • 4 个回复 • 206 次浏览 • 2019-03-06 10:19\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 92 次浏览 • 2019-03-05 11:11 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 3 人关注 • 0 个回复 • 114 次浏览 • 2019-03-04 19:06 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 6 人关注 • 5 个回复 • 393 次浏览 • 2019-03-04 11:33\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 2 人关注 • 0 个回复 • 114 次浏览 • 2019-02-27 11:07 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 143 次浏览 • 2019-02-26 18:28 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 15 人关注 • 4 个回复 • 3997 次浏览 • 2019-02-25 11:51\t\t\t\t\n",
      " •  来自相关主题\n",
      "回复了问题 • 3 人关注 • 1 个回复 • 119 次浏览 • 2019-02-23 22:14\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 13 人关注 • 11 个回复 • 981 次浏览 • 2019-02-22 21:18\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 110 次浏览 • 2019-02-22 12:17 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 135 次浏览 • 2019-02-21 17:34 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 21 人关注 • 16 个回复 • 2479 次浏览 • 2019-02-20 01:11\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 116 次浏览 • 2019-02-19 15:28 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 2 人关注 • 1 个回复 • 126 次浏览 • 2019-02-19 11:33\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 2 人关注 • 0 个回复 • 165 次浏览 • 2019-02-18 11:47 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 2 人关注 • 1 个回复 • 274 次浏览 • 2019-02-15 15:05\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 2 人关注 • 1 个回复 • 94 次浏览 • 2019-02-15 14:58\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 169 次浏览 • 2019-02-15 10:42 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 2 人关注 • 1 个回复 • 119 次浏览 • 2019-02-14 17:39\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 165 次浏览 • 2019-02-12 22:11 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 214 次浏览 • 2019-02-01 17:43 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "回复了问题 • 2 人关注 • 1 个回复 • 592 次浏览 • 2019-01-31 17:15\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 320 次浏览 • 2019-01-31 12:01 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 2 人关注 • 0 个回复 • 225 次浏览 • 2019-01-30 17:07 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 3 人关注 • 0 个回复 • 226 次浏览 • 2019-01-30 11:59 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 275 次浏览 • 2019-01-28 12:15 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 18 人关注 • 1 个回复 • 1290 次浏览 • 2019-01-27 14:11\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 206 次浏览 • 2019-01-24 11:57 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 264 次浏览 • 2019-01-24 11:51 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 232 次浏览 • 2019-01-23 11:48 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 241 次浏览 • 2019-01-23 11:26 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 164 次浏览 • 2019-01-23 10:49 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "贡献\n",
      "回复了问题 • 5 人关注 • 5 个回复 • 347 次浏览 • 2019-01-23 10:27\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 207 次浏览 • 2019-01-22 14:52 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 215 次浏览 • 2019-01-22 14:43 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 182 次浏览 • 2019-01-21 10:26 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 280 次浏览 • 2019-01-18 12:37 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 1 人关注 • 0 个回复 • 242 次浏览 • 2019-01-18 12:31 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "发起了问题 • 2 人关注 • 0 个回复 • 402 次浏览 • 2019-01-17 17:35 \n",
      "\t\t\t\t\n",
      " •  来自相关主题\n",
      "\n",
      "\t\t\t\t\t\t\t面经大赛\n",
      "\t\t\t\t\t\t\n",
      "返回顶部\n",
      "\r\n",
      "\t\t\t\t\tdocument.write(unescape(\"%3Cspan id='cnzz_stat_icon_1259748782'%3E%3C/span%3E%3Cscript src='\" + \"https://s11.cnzz.com/z_stat.php%3Fid%3D1259748782%26show%3Dpic' type='text/javascript'%3E%3C/script%3E\"));\r\n",
      "\t\t\t\t\n",
      "\r\n",
      "\t\t\t\t\t\t\r\n",
      "\t\t\t\t\t\t\r\n",
      "\t\t\t\t\t\n",
      "\r\n",
      "\t\t\t\t\t\t\r\n",
      "\t\t\t\t\t\t\r\n",
      "\t\t\t\t\t\n",
      "\r\n",
      "\t\t\t\t\t\t\r\n",
      "\t\t\t\t\t\t\r\n",
      "\t\t\t\t\t\n"
     ]
    }
   ],
   "source": [
    "def _unpack(spans):\n",
    "    return [val.text_content() for val in spans]\n",
    "contents = _unpack(spans)\n",
    "for content in contents:\n",
    "    print content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions = doc.findall('.//h4')\n",
    "len(questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\t\t\t\t\t\t励志！充满干货的AI面经：纯电力员工如何成功转行NLP并薪资翻倍\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t【阿里巴巴-天猫进出口事业部】2019春季-测试开发工程师-招实习生啦！！！\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t【面经】一个机械转行算法的菜逼应届生如何进华为\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\tAI 面经：我是如何从机械行业本科社招成功转行NLP并薪资翻倍的\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t【大量招】阿里巴巴-天猫进出口技术部-2020届春季实习生招聘\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t【阿里巴巴-新零售事业群】2019春季算法实习生内推\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t【OCR技术】大批量生成文字训练集\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t非科班生拿BAT算法SP offer面经（阿里巴巴，腾讯……）\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t浅谈决策树、GBDT、LightGBM\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\tAI offer面经：薪资近乎翻倍，题库里的xgboost笔记看了不下十遍\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t七月在线AI就业班 申请信息填报表\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t红包问答（已开奖）：下面有关JVM内存，说法错误的是？\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t一文详解机器学习中最好用的提升方法：Boosting 与 AdaBoost\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t写课程笔记，全额返学费——[无人驾驶实战]\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t红包问答（已开奖）：随机森林如何处理缺失值？\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t必备收藏！8500+公开代码论文，950多项机器学习任务最优结果汇总\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t不看后悔！2019年人工智能行业25大趋势\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t面试高频题：给定N个数的间距最大值\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t自动驾驶课程学员资料共享\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t红包问答（已开奖）：哪些机器学习算法不需要做归一化处理？\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t再见了，快递员！北京打响第一枪！\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t必读！2018最具突破性计算机视觉论文Top 10\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t我的算法岗校招面经：微软、谷歌、阿里、头条、地平线、网易游戏等\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t特朗普终于顾不得美国人就业，准备举国搞人工智能了\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t我有门课选错，可以退款吗\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t程序员面试谈薪指南\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t悬赏话题：35 岁是不是程序员的一个坎？\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t春节红包活动\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t11万份测试告诉你，今年学什么编程语言才能找到好工作\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t【收藏】机器学习开源框架大汇总，总有一款适合你\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t如何通俗理解LSTM网络(台大版)\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t求解答：xgboost 命令行训练和python接口训练得到的模型不一样\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\tnlp笔试题\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t指南：为什么学ai、ai薪资水平和2019年ai就业前景分析\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t推荐系统综合项目解析与特征处理\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t在线推荐系统课期末项目解法和分析\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t裤子换裙子，就问你GAN的这波操作秀不秀\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t9元尊享【七月在线VIP年会员】\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t还在为数据清洗抓狂？这里有一个简单实用的清洗代码集\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t程序员锁死服务器搅黄游戏，600万打水漂，创始人负债数百万！\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t全球最厉害的14位程序员！你都认识吗？\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t程序员吐槽：老板从BAT挖来的同事，月薪4万还不如自己\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t北京上海某知名国际互联网经济型连锁酒店正在招聘java岗位中！！！\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t后台开发_国企研究所 + IT大厂\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t8种寻找机器学习数据集的方法 | 附数据集资源\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t自动驾驶之路已走了多远？一文读懂研究现状\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t干货收藏丨50个史上最佳机器学习公共数据集\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t我是如何一步步拿下 Google Offer 的？\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t计算机专业毕业？这是给你的职业建议\n",
      "\t\t\t\t\t\n",
      "\n",
      "\t\t\t\t\t\t七月在线【推荐系统就业班】申请信息填报表\n",
      "\t\t\t\t\t\n"
     ]
    }
   ],
   "source": [
    "contents = _unpack(questions)\n",
    "for content in contents:\n",
    "    print content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.7 解析XML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import objectify\n",
    "\n",
    "path = './4/data/movies.xml'\n",
    "parsed = objectify.parse(open(path))\n",
    "root = parsed.getroot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "\n",
    "skip_fields = ['PARENT_SEQ', 'INDICATOR_SEQ',\n",
    "               'DESIRED_CHANGE', 'DECIMAL_PLACES']\n",
    "\n",
    "for elt in  root.movie:\n",
    "    el_data = {}\n",
    "    for child in elt.getchildren():\n",
    "        if child.tag in skip_fields:\n",
    "            continue\n",
    "        el_data[child.tag] = child.pyval\n",
    "    data.append(el_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>episodes</th>\n",
       "      <th>format</th>\n",
       "      <th>rating</th>\n",
       "      <th>stars</th>\n",
       "      <th>type</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Talk about a US-Japan war</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DVD</td>\n",
       "      <td>PG</td>\n",
       "      <td>10</td>\n",
       "      <td>War, Thriller</td>\n",
       "      <td>2003.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A schientific fiction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DVD</td>\n",
       "      <td>R</td>\n",
       "      <td>8</td>\n",
       "      <td>Anime, Science Fiction</td>\n",
       "      <td>1989.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Vash the Stampede!</td>\n",
       "      <td>4.0</td>\n",
       "      <td>DVD</td>\n",
       "      <td>PG</td>\n",
       "      <td>10</td>\n",
       "      <td>Anime, Action</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Viewable boredom</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VHS</td>\n",
       "      <td>PG</td>\n",
       "      <td>2</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 description  episodes format rating  stars  \\\n",
       "0  Talk about a US-Japan war       NaN    DVD     PG     10   \n",
       "1      A schientific fiction       NaN    DVD      R      8   \n",
       "2         Vash the Stampede!       4.0    DVD     PG     10   \n",
       "3           Viewable boredom       NaN    VHS     PG      2   \n",
       "\n",
       "                     type    year  \n",
       "0           War, Thriller  2003.0  \n",
       "1  Anime, Science Fiction  1989.0  \n",
       "2           Anime, Action     NaN  \n",
       "3                  Comedy     NaN  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perf = DataFrame(data)\n",
    "perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Element collection at 0x9dde748>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'New Arrivals'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root.get('shelf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# root.text\n",
    "len(root)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 二进制格式的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame = pd.read_csv('./4/data/data1.csv')\n",
    "frame\n",
    "frame.to_pickle('./4/data/frame_pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b   c   d message\n",
       "0  1   2   3   4   hello\n",
       "1  5   6   7   8   world\n",
       "2  9  10  11  12     foo"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_pickle('./4/data/frame_pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 使用HDF5格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<class 'pandas.io.pytables.HDFStore'>\n",
       "File path: ./4/data/mydata.h5"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store = pd.HDFStore('./4/data/mydata.h5')\n",
    "store['obj1'] = frame\n",
    "store['obj1_col'] = frame['a']\n",
    "store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>foo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b   c   d message\n",
       "0  1   2   3   4   hello\n",
       "1  5   6   7   8   world\n",
       "2  9  10  11  12     foo"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store['obj1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "store.close()\n",
    "os.remove('./4/data/mydata.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HTML与API交互"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "url = 'https://api.github.com/repos/pydata/pandas/milestones/28/labels'\n",
    "resp = requests.get(url)\n",
    "resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'description': 'Talk about a US-Japan war',\n",
       "  'format': 'DVD',\n",
       "  'rating': 'PG',\n",
       "  'stars': 10,\n",
       "  'type': 'War, Thriller',\n",
       "  'year': 2003},\n",
       " {'description': 'A schientific fiction',\n",
       "  'format': 'DVD',\n",
       "  'rating': 'R',\n",
       "  'stars': 8,\n",
       "  'type': 'Anime, Science Fiction',\n",
       "  'year': 1989},\n",
       " {'description': 'Vash the Stampede!',\n",
       "  'episodes': 4,\n",
       "  'format': 'DVD',\n",
       "  'rating': 'PG',\n",
       "  'stars': 10,\n",
       "  'type': 'Anime, Action'},\n",
       " {'description': 'Viewable boredom',\n",
       "  'format': 'VHS',\n",
       "  'rating': 'PG',\n",
       "  'stars': 2,\n",
       "  'type': 'Comedy'}]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "print resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>episodes</th>\n",
       "      <th>format</th>\n",
       "      <th>rating</th>\n",
       "      <th>stars</th>\n",
       "      <th>type</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Talk about a US-Japan war</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DVD</td>\n",
       "      <td>PG</td>\n",
       "      <td>10</td>\n",
       "      <td>War, Thriller</td>\n",
       "      <td>2003.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A schientific fiction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DVD</td>\n",
       "      <td>R</td>\n",
       "      <td>8</td>\n",
       "      <td>Anime, Science Fiction</td>\n",
       "      <td>1989.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Vash the Stampede!</td>\n",
       "      <td>4.0</td>\n",
       "      <td>DVD</td>\n",
       "      <td>PG</td>\n",
       "      <td>10</td>\n",
       "      <td>Anime, Action</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Viewable boredom</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VHS</td>\n",
       "      <td>PG</td>\n",
       "      <td>2</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 description  episodes format rating  stars  \\\n",
       "0  Talk about a US-Japan war       NaN    DVD     PG     10   \n",
       "1      A schientific fiction       NaN    DVD      R      8   \n",
       "2         Vash the Stampede!       4.0    DVD     PG     10   \n",
       "3           Viewable boredom       NaN    VHS     PG      2   \n",
       "\n",
       "                     type    year  \n",
       "0           War, Thriller  2003.0  \n",
       "1  Anime, Science Fiction  1989.0  \n",
       "2           Anime, Action     NaN  \n",
       "3                  Comedy     NaN  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "issue_labels = DataFrame(data)\n",
    "issue_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.数据库相关操作"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 sqlite数据库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "\n",
    "query = \"\"\"\n",
    "CREATE TABLE test(\n",
    "a VARCHAR(20),\n",
    "b VARCHAR(20),\n",
    "c REAL,\n",
    "d INTEGER\n",
    ");\n",
    "\"\"\"\n",
    "\n",
    "con = sqlite3.connect(\":memory:\")\n",
    "con.execute(query)\n",
    "con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [('Atlanta', 'Georgia', 1.25, 6),\n",
    "       ('Tallahassee', 'Florida', 2.6, 3),\n",
    "       ('Sacramento', 'California', 1.7, 5)]\n",
    "stmt = \"INSERT INTO test VALUES(?, ?, ?, ?)\"\n",
    "\n",
    "con.executemany(stmt, data)\n",
    "con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Atlanta', u'Georgia', 1.25, 6),\n",
       " (u'Tallahassee', u'Florida', 2.6, 3),\n",
       " (u'Sacramento', u'California', 1.7, 5)]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cursor = con.execute('select * from test')\n",
    "rows = cursor.fetchall()\n",
    "rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(('a', None, None, None, None, None, None),\n",
       " ('b', None, None, None, None, None, None),\n",
       " ('c', None, None, None, None, None, None),\n",
       " ('d', None, None, None, None, None, None))"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cursor.description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Atlanta</td>\n",
       "      <td>Georgia</td>\n",
       "      <td>1.25</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tallahassee</td>\n",
       "      <td>Florida</td>\n",
       "      <td>2.60</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sacramento</td>\n",
       "      <td>California</td>\n",
       "      <td>1.70</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             a           b     c  d\n",
       "0      Atlanta     Georgia  1.25  6\n",
       "1  Tallahassee     Florida  2.60  3\n",
       "2   Sacramento  California  1.70  5"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrame(rows, columns=zip(*cursor.description)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Atlanta</td>\n",
       "      <td>Georgia</td>\n",
       "      <td>1.25</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tallahassee</td>\n",
       "      <td>Florida</td>\n",
       "      <td>2.60</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sacramento</td>\n",
       "      <td>California</td>\n",
       "      <td>1.70</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             a           b     c  d\n",
       "0      Atlanta     Georgia  1.25  6\n",
       "1  Tallahassee     Florida  2.60  3\n",
       "2   Sacramento  California  1.70  5"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas.io.sql as sql\n",
    "sql.read_sql('select * from  test',con)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
